#!/usr/bin/env python
#encoding:utf8

'''
web client
'''

import socket
import urllib2

###################################################################################
class web(object):
	'''
	An url web page
	'''
	def __init__(self, url, encoding= None, timeout= socket._GLOBAL_DEFAULT_TIMEOUT):
		self.__url		= url
		self.__encoding	= encoding
		self.__timeout	= timeout
	
	def __str__(self):
		return self.text

	def _get_content_charset(self, response):
		'''
		Get charset of the web page
		'''
		content_type	= response.headers.get('content-type')
		if content_type:
			splits		= content_type.split('=')
			if len(splits)==2:
				charset	= splits[1]
				return charset
		return None
	
	@staticmethod
	def urlopen(url, data= None, timeout= socket._GLOBAL_DEFAULT_TIMEOUT):
		'''
		Open url and return a response
		'''
		# be sure url starts with http://
		url	= url.lower()
		if not url.startswith('http://'):
			url	= 'http://' + url
	
		try:
			request		= urllib2.Request(url)
			request.add_header('User-Agent','Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US)')
			return urllib2.urlopen(request, data, timeout= timeout)
		except ValueError as ex:
			pass

	@property
	def url(self):
		return self.__url

	@property
	def data(self):
		'''
		Get raw data from the url
		'''
		if not hasattr(self, '__data'):
			# open connection and read all data
			response	= web.urlopen(self.__url, None, self.__timeout)
			self.__data	= response.read()
			response.close()

		return self.__data

	@property
	def text(self):
		'''
		Get text from the url
		'''
		if not hasattr(self, '__text'):
			response	= web.urlopen(self.__url, None, self.__timeout)
			assert response, 'response should not be of NoneType'
			if not hasattr(self, '__data'):
				self.__data	= response.read()

			# get charset of the web page
			if not self.__encoding:
				self.__encoding	= self._get_content_charset(response) or 'utf8'

			# decode
			self.__text		= self.__data.decode(self.__encoding)

			response.close()

		return self.__text
	
	def save(self, filename):
		'''
		Save the web content to local disk
		'''
		import os
		# fetch data
		data	= self.data

		# make sure the directory exist
		dirname	= os.path.dirname(filename)
		if not os.path.exists(dirname):
			os.makedirs(dirname)

		# write file
		open(filename, 'wb').write(data)


###################################################################################

if __name__=='__main__':
	urls		= ['www.baidu.com', 'python.org']

	try:
		for url in urls:
			page = web(url)
			print(page.text)
	except Exception as ex:
		print 'error occurred:', ex
